from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
from os import walk

# step3:
# add the original list length of the terms
def buildingTrainingFile_step3():
    termListDict = {}
    ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2_UPP-5_1%_LexiconTermID_Term_ListLength_sortedByAlphabeticalOrder"
    ifh0 = open(ifn0,"r")
    for line in ifh0.readlines():
        lineElements = line.strip().split(" ")
        term = lineElements[1]
        listLength = int(lineElements[2])
        termListDict[term] = listLength 
    print "len(termListDict):",len(termListDict)
    ifh0.close()
    
    ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/trainingFile_20140908_step3"
    ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/trainingFile_20140908_step2"
    ifh = open(ifn,"r")
    ofh = open(ofn,"w")
    for line in ifh.readlines():
        lineElements = line.strip().split(":")
        queryTerms = lineElements[1].split(" ")
        completeOutputLine = line.strip() + " "
        extendedOutputLine = ""
        for queryTerm in queryTerms:
            if queryTerm in termListDict:
                extendedOutputLine += str(termListDict[queryTerm]) + " "
            else:
                extendedOutputLine += "0" + " "
        completeOutputLine = line.strip() + ":" + extendedOutputLine
        ofh.write(completeOutputLine.strip() + "\n")
    ifh.close()
    ofh.close()
    print "Overall:"
    print "ifn:",ifn
    print "ofn:",ofn

# step2:
# add the term list length from the pruned index
def buildingTrainingFile_step2():
    termListDict = {}
    # load the term list length
    ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/debug_term_related"
    ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/trainingFile_20140908_step2"
    ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/trainingFile_20140908_step1"
    
    ifh0 = open(ifn0,"r")
    for line in ifh0.readlines():
        lineElements = line.strip().split(" ")
        term = lineElements[0]
        listLength = int(lineElements[1])
        termListDict[term] = listLength 
    print "len(termListDict):",len(termListDict)
    ifh0.close()
    
    ifh = open(ifn,"r")
    ofh = open(ofn,"w")
    for line in ifh.readlines():
        lineElements = line.strip().split(":")
        queryTerms = lineElements[1].split(" ")
        completeOutputLine = line.strip() + " "
        extendedOutputLine = ""
        for queryTerm in queryTerms:
            if queryTerm in termListDict:
                extendedOutputLine += str(termListDict[queryTerm]) + " "
            else:
                extendedOutputLine += "0" + " "
        completeOutputLine = line.strip() + ":" + extendedOutputLine
        ofh.write(completeOutputLine.strip() + "\n")
    ifh.close()
    ofh.close()
    print "Overall:"
    print "ifn:",ifn
    print "ofn:",ofn

# step1:
def buildingTrainingFile_step1():
    ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/trainingFile_20140908_step1"
    ifn = "/home/vgc/wei/workspace/NYU_IRTK/data/GOV2_100KQueries_head_95K"
    ifh = open(ifn,"r")
    ofh = open(ofn,"w")
    for line in ifh.readlines():
        lineElements = line.strip().split(":")
        queryLength = len(lineElements[1].strip().split(" "))
        ofh.write(line.strip() + ":" + str(queryLength) + "\n")
    ifh.close()
    ofh.close()
    print "Overall:"
    print "ifn:",ifn
    print "ofn:",ofn


print "Begins..."
buildingTrainingFile_step1()
buildingTrainingFile_step2()
print "Ends."
exit(1)

print "Begins..."
ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/trainingFile_20140915_step8_extended.arff"
ofh = open(ofn,"w")

unprunedQueryTermsSamplingValuesDict = {}
ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/Unpruned_100%_queryTermsSamplingValues_20140915_fixed"
ifh = open(ifn3,"r")
l = ifh.readline()
while l:
    le = l.strip().split(" ")
    contentLine = ""
    term = le[0]
    for value in le[1:]:
        contentLine += value + ","
    unprunedQueryTermsSamplingValuesDict[term] = contentLine[:-1]
    l = ifh.readline()
ifh.close()
print "Overall:"
print "len(unprunedQueryTermsSamplingValuesDict):",len(unprunedQueryTermsSamplingValuesDict)

upp5_1PercentQueryTermsSamplingValuesDict = {}
ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/UPP-5_1%_queryTermsSamplingValues_20140915_fixed"
ifh = open(ifn4,"r")
l = ifh.readline()
while l:
    le = l.strip().split(" ")
    contentLine = ""
    term = le[0]
    for value in le[1:]:
        contentLine += value + ","
    upp5_1PercentQueryTermsSamplingValuesDict[term] = contentLine[:-1]
    l = ifh.readline()    
ifh.close()
print "Overall:"
print "len(upp5_1PercentQueryTermsSamplingValuesDict):",len(upp5_1PercentQueryTermsSamplingValuesDict)
print "upp5_1PercentQueryTermsSamplingValuesDict['zydeco']:",upp5_1PercentQueryTermsSamplingValuesDict['zydeco']

qidWithTermListDict = {}
ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/Unpruned_OR_100%_auxFile_20140913"
ifh = open(ifn1,"r")
l = ifh.readline()
while l:
    if l.strip().startswith("qid"):
        qid = l.strip().split(" ")[1]
        # print qid
        termList = []
        elements = ifh.readline().strip().split(" ")
        for term in elements[1:]:
            termList.append(term)
        qidWithTermListDict[qid] = termList
    l = ifh.readline()
print "len(qidWithTermListDict):",len(qidWithTermListDict)
print "sth:",qidWithTermListDict["14104"]
ifh.close()

ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/trainingFile_20140913_step8.arff"
ifh = open(ifn2,"r")
print "ifn2:",ifn2
l = ifh.readline()
while l:
    le = l.strip().split(",")
    if len(le) == 37:
        # print "l:",l
        # print "check point:",len(l.split(","))
        outputLine = ""
        qid = le[0]
        # print "qid:",qid
        part1 = ""
        part2 = ""
        part2b = ""
        part3 = ""
        part3b = ""
        part4 = ""
        part5 = ""
        for i in range(0,2):
            part1 += le[i] + ","
        # print "part1:",part1
        # print len(part1.strip().split(","))
        outputLine += part1
        for i in range(2,12):
            part2 += le[i] + ","
        # print "part2:",part2
        # print len(part2.strip().split(","))
        outputLine += part2
        
        number = 0
        if len(qidWithTermListDict[qid]) > 10:
            number = 10
        else:
            number = len(qidWithTermListDict[qid])
        for term in qidWithTermListDict[qid][:number]:
            if term not in upp5_1PercentQueryTermsSamplingValuesDict:
                part2b += "?,?,?,?,?,?,?,?,?,?" + ","
            else:
                part2b += upp5_1PercentQueryTermsSamplingValuesDict[term] + ","
        for i in range(0,10-number):
            part2b += "?,?,?,?,?,?,?,?,?,?" + ","
        # print "part2b:",part2b
        # print len(part2b.strip().split(","))
        outputLine += part2b
        
        
        for i in range(12,22):
            part3 += le[i] + ","
        # print "part3:",part3
        # print len(part3.strip().split(","))
        outputLine += part3

        
        number = 0
        if len(qidWithTermListDict[qid]) > 10:
            number = 10
        else:
            number = len(qidWithTermListDict[qid])
        for term in qidWithTermListDict[qid][:number]:
            if term not in unprunedQueryTermsSamplingValuesDict:
                part3b += "?,?,?,?,?,?,?,?,?,?" + ","
            else:
                part3b += unprunedQueryTermsSamplingValuesDict[term] + ","
        for i in range(0,10-number):
            part3b += "?,?,?,?,?,?,?,?,?,?" + ","
        # print "part2b:",part2b
        # print len(part2b.strip().split(","))
        outputLine += part3b
        
        
        for i in range(22,32):
            part4 += le[i] + ","
        # print "part4:",part4
        # print len(part4.strip().split(","))
        outputLine += part4
        
        for i in range(32,37):    
            part5 += le[i] + ","
        # print "part5:",part5
        # print len(part5.strip().split(","))
        outputLine += part5
        
        outputLine = outputLine[:-1] + "\n"
        # print "outputLine:",outputLine
        # print "check point:",len(outputLine.split(","))
        ofh.write(outputLine)
    else:
        ofh.write(l)
    l = ifh.readline()
print "Overall:"
print "ifn1:",ifn1
print "ifn2:",ifn2
print "ifn3:",ifn3
print "ifn4:",ifn4
print "ofn:",ofn
print "End."














